* Replication code for Figures 1 and 2 of "A Computational Model for Identifying Delegating and Constraining Provisions in European Union Laws"
* Fabio Franchino, Marta Migliorati, Giovanni Pagano and Valerio Vignoli
* European Union Politics

version 18
clear all

use annotations.dta, clear

********** Table A6: Performance metrics: rule-based application vs. annotators classification
* Note: M denotes rule-based application variables, AN denotes annotators classification

*** Delegation to Member States
* True positives
count if Mdel_ms == 1 & ANdel_ms == 1 
scalar TP = r(N)
* False positives
count if Mdel_ms == 1 & ANdel_ms == 0
scalar FP = r(N)
* True negatives
count if Mdel_ms == 0 & ANdel_ms == 0 
scalar TN = r(N)
* False negatives
count if Mdel_ms == 0 & ANdel_ms == 1
scalar FN = r(N)
* Precision
scalar Precision = TP / (TP + FP)
scalar Recall = TP / (TP + FN)
scalar F1 = (2 * Precision * Recall) / (Precision + Recall)
scalar MCC = ((TP * TN) - (FP * FN)) / sqrt((TP + FP) * (TP + FN) * (TN + FP) * (TN + FN))

* Create matrix
matrix P = Precision
matrix R = Recall
matrix F = F1
matrix M = MCC

matrix T = (P, R , F , M)
mat list T

*** Constraint to Member States
* True positives
count if Mcon_ms == 1 & ANcon_ms == 1 
scalar TP = r(N)
* False positives
count if Mcon_ms == 1 & ANcon_ms == 0
scalar FP = r(N)
* True negatives
count if Mcon_ms == 0 & ANcon_ms == 0 
scalar TN = r(N)
* False negatives
count if Mcon_ms == 0 & ANcon_ms == 1
scalar FN = r(N)
* Precision
scalar Precision = TP / (TP + FP)
scalar Recall = TP / (TP + FN)
scalar F1 = (2 * Precision * Recall) / (Precision + Recall)
scalar MCC = ((TP * TN) - (FP * FN)) / sqrt((TP + FP) * (TP + FN) * (TN + FP) * (TN + FN))

* Create matrix
matrix P = Precision
matrix R = Recall
matrix F = F1
matrix M = MCC

matrix T1 = (P, R , F , M)

* Merge matrix
matrix T = (T \ T1)
mat list T

*** Soft obligations to Member States
* True positives
count if Mso_ms == 1 & ANso_ms == 1 
scalar TP = r(N)
* False positives
count if Mso_ms == 1 & ANso_ms == 0
scalar FP = r(N)
* True negatives
count if Mso_ms == 0 & ANso_ms == 0 
scalar TN = r(N)
* False negatives
count if Mso_ms == 0 & ANso_ms == 1
scalar FN = r(N)
* Precision
scalar Precision = TP / (TP + FP)
scalar Recall = TP / (TP + FN)
scalar F1 = (2 * Precision * Recall) / (Precision + Recall)
scalar MCC = ((TP * TN) - (FP * FN)) / sqrt((TP + FP) * (TP + FN) * (TN + FP) * (TN + FN))

* Create matrix
matrix P = Precision
matrix R = Recall
matrix F = F1
matrix M = MCC

matrix T1 = (P, R , F , M)

* Merge matrix
matrix T = (T \ T1)
mat list T

*** Delegation to National Competent Authorities
* True positives
count if Mdel_nca == 1 & ANdel_nca == 1 
scalar TP = r(N)
* False positives
count if Mdel_nca == 1 & ANdel_nca == 0
scalar FP = r(N)
* True negatives
count if Mdel_nca == 0 & ANdel_nca == 0 
scalar TN = r(N)
* False negatives
count if Mdel_nca == 0 & ANdel_nca == 1
scalar FN = r(N)
* Precision
scalar Precision = TP / (TP + FP)
scalar Recall = TP / (TP + FN)
scalar F1 = (2 * Precision * Recall) / (Precision + Recall)
scalar MCC = ((TP * TN) - (FP * FN)) / sqrt((TP + FP) * (TP + FN) * (TN + FP) * (TN + FN))

* Create matrix
matrix P = Precision
matrix R = Recall
matrix F = F1
matrix M = MCC

matrix T1 = (P, R , F , M)

* Merge matrix
matrix T = (T \ T1)
mat list T

*** Constraint to National Competent Authorities
* True positives
count if Mcon_nca == 1 & ANcon_nca == 1 
scalar TP = r(N)
* False positives
count if Mcon_nca == 1 & ANcon_nca == 0
scalar FP = r(N)
* True negatives
count if Mcon_nca == 0 & ANcon_nca == 0 
scalar TN = r(N)
* False negatives
count if Mcon_nca == 0 & ANcon_nca == 1
scalar FN = r(N)
* Precision
scalar Precision = TP / (TP + FP)
scalar Recall = TP / (TP + FN)
scalar F1 = (2 * Precision * Recall) / (Precision + Recall)
scalar MCC = ((TP * TN) - (FP * FN)) / sqrt((TP + FP) * (TP + FN) * (TN + FP) * (TN + FN))

* Create matrix
matrix P = Precision
matrix R = Recall
matrix F = F1
matrix M = MCC

matrix T1 = (P, R , F , M)

* Merge matrix
matrix T = (T \ T1)
mat list T

*** Soft obligations to National Competent Authorities
* True positives
count if Mso_nca == 1 & ANso_nca == 1 
scalar TP = r(N)
* False positives
count if Mso_nca == 1 & ANso_nca == 0
scalar FP = r(N)
* True negatives
count if Mso_nca == 0 & ANso_nca == 0 
scalar TN = r(N)
* False negatives
count if Mso_nca == 0 & ANso_nca == 1
scalar FN = r(N)
* Precision
scalar Precision = TP / (TP + FP)
scalar Recall = TP / (TP + FN)
scalar F1 = (2 * Precision * Recall) / (Precision + Recall)
scalar MCC = ((TP * TN) - (FP * FN)) / sqrt((TP + FP) * (TP + FN) * (TN + FP) * (TN + FN))

* Create matrix
matrix P = Precision
matrix R = Recall
matrix F = F1
matrix M = MCC

matrix T1 = (P, R , F , M)

* Merge matrix
matrix T = (T \ T1)
mat list T

*** Agenda power of the Commission
* True positives
count if Magenda == 1 & ANagenda == 1 
scalar TP = r(N)
* False positives
count if Magenda == 1 & ANagenda == 0
scalar FP = r(N)
* True negatives
count if Magenda == 0 & ANagenda == 0 
scalar TN = r(N)
* False negatives
count if Magenda == 0 & ANagenda == 1
scalar FN = r(N)
* Precision
scalar Precision = TP / (TP + FP)
scalar Recall = TP / (TP + FN)
scalar MCC = ((TP * TN) - (FP * FN)) / sqrt((TP + FP) * (TP + FN) * (TN + FP) * (TN + FN))

* Create matrix
matrix P = Precision
matrix R = Recall
matrix F = F1
matrix M = MCC

matrix T1 = (P, R , F , M)

* Merge matrix
matrix T = (T \ T1)
mat list T

*** Delegation to Commission
* True positives
count if Mdel_com == 1 & ANdel_com == 1 
scalar TP = r(N)
* False positives
count if Mdel_com == 1 & ANdel_com == 0
scalar FP = r(N)
* True negatives
count if Mdel_com == 0 & ANdel_com == 0 
scalar TN = r(N)
* False negatives
count if Mdel_com == 0 & ANdel_com == 1
scalar FN = r(N)
* Precision
scalar Precision = TP / (TP + FP)
scalar Recall = TP / (TP + FN)
scalar MCC = ((TP * TN) - (FP * FN)) / sqrt((TP + FP) * (TP + FN) * (TN + FP) * (TN + FN))

* Create matrix
matrix P = Precision
matrix R = Recall
matrix F = F1
matrix M = MCC

matrix T1 = (P, R , F , M)

* Merge matrix
matrix T = (T \ T1)
mat list T

*** Constraint to Commission
* True positives
count if Mcon_com == 1 & ANcon_com == 1 
scalar TP = r(N)
* False positives
count if Mcon_com == 1 & ANcon_com == 0
scalar FP = r(N)
* True negatives
count if Mcon_com == 0 & ANcon_com == 0 
scalar TN = r(N)
* False negatives
count if Mcon_com == 0 & ANcon_com == 1
scalar FN = r(N)
* Precision
scalar Precision = TP / (TP + FP)
scalar Recall = TP / (TP + FN)
scalar MCC = ((TP * TN) - (FP * FN)) / sqrt((TP + FP) * (TP + FN) * (TN + FP) * (TN + FN))

* Create matrix
matrix P = Precision
matrix R = Recall
matrix F = F1
matrix M = MCC

matrix T1 = (P, R , F , M)

* Merge matrix
matrix T = (T \ T1)
mat list T

*** Soft implementation of Commission
* True positives
count if Msi_com == 1 & ANsi_com == 1 
scalar TP = r(N)
* False positives
count if Msi_com == 1 & ANsi_com == 0
scalar FP = r(N)
* True negatives
count if Msi_com == 0 & ANsi_com == 0 
scalar TN = r(N)
* False negatives
count if Msi_com == 0 & ANsi_com == 1
scalar FN = r(N)
* Precision
scalar Precision = TP / (TP + FP)
scalar Recall = TP / (TP + FN)
scalar F1 = (2 * Precision * Recall) / (Precision + Recall)
scalar MCC = ((TP * TN) - (FP * FN)) / sqrt((TP + FP) * (TP + FN) * (TN + FP) * (TN + FN))

* Create matrix
matrix P = Precision
matrix R = Recall
matrix F = F1
matrix M = MCC

matrix T1 = (P, R , F , M)

* Merge matrix
matrix T = (T \ T1)
mat list T

*** Delegation to Agencies
* True positives
count if Mdel_age == 1 & ANdel_age == 1 
scalar TP = r(N)
* False positives
count if Mdel_age == 1 & ANdel_age == 0
scalar FP = r(N)
* True negatives
count if Mdel_age == 0 & ANdel_age == 0 
scalar TN = r(N)
* False negatives
count if Mdel_age == 0 & ANdel_age == 1
scalar FN = r(N)
* Precision
scalar Precision = TP / (TP + FP)
scalar Recall = TP / (TP + FN)
scalar F1 = (2 * Precision * Recall) / (Precision + Recall)
scalar MCC = ((TP * TN) - (FP * FN)) / sqrt((TP + FP) * (TP + FN) * (TN + FP) * (TN + FN))

* Create matrix
matrix P = Precision
matrix R = Recall
matrix F = F1
matrix M = MCC

matrix T1 = (P, R , F , M)

* Merge matrix
matrix T = (T \ T1)
mat list T

*** Constraint to Agencies
* True positives
count if Mcon_age == 1 & ANcon_age == 1 
scalar TP = r(N)
* False positives
count if Mcon_age == 1 & ANcon_age == 0
scalar FP = r(N)
* True negatives
count if Mcon_age == 0 & ANcon_age == 0 
scalar TN = r(N)
* False negatives
count if Mcon_age == 0 & ANcon_age == 1
scalar FN = r(N)
* Precision
scalar Precision = TP / (TP + FP)
scalar Recall = TP / (TP + FN)
scalar F1 = (2 * Precision * Recall) / (Precision + Recall)
scalar MCC = ((TP * TN) - (FP * FN)) / sqrt((TP + FP) * (TP + FN) * (TN + FP) * (TN + FN))

* Create matrix
matrix P = Precision
matrix R = Recall
matrix F = F1
matrix M = MCC

matrix T1 = (P, R , F , M)

* Merge matrix
matrix T = (T \ T1)
mat list T

*** Soft implementation of Agencies
* True positives
count if Msi_age == 1 & ANsi_age == 1 
scalar TP = r(N)
* False positives
count if Msi_age == 1 & ANsi_age == 0
scalar FP = r(N)
* True negatives
count if Msi_age == 0 & ANsi_age == 0 
scalar TN = r(N)
* False negatives
count if Msi_age == 0 & ANsi_age == 1
scalar FN = r(N)
* Precision
scalar Precision = TP / (TP + FP)
scalar Recall = TP / (TP + FN)
scalar F1 = (2 * Precision * Recall) / (Precision + Recall)
scalar MCC = ((TP * TN) - (FP * FN)) / sqrt((TP + FP) * (TP + FN) * (TN + FP) * (TN + FN))

* Create matrix
matrix P = Precision
matrix R = Recall
matrix F = F1
matrix M = MCC

matrix T1 = (P, R , F , M)

* Merge matrix
matrix T = (T \ T1)
mat list T

* Names
matrix colnames T = Precision Recall F1 MCC
matrix rownames T = del_ms con_ms so_ms del_nca con_nca so_nca agenda del_com si_com con_com del_age si_age con_age
mat list T
svmat T, names(col)
replace Precision = 100 * round(Precision,.0001)
replace Recall = 100 * round(Recall,.0001)
replace F1 = 100 * round(F1,.0001)
replace MCC = 100 * round(MCC,.0001)

asdoc list Precision Recall F1 MCC if F1!=., replace save(TableA6manualAN_model.doc) title(Table A6: Performance metrics: rule-based application vs. annotators classification)  dec(2)
